#
# This is a standalone github action that runs promptfoo against a change.
# You'll have to edit it to use the appropriate branches and prompts.
#
# For an easier to use packaged action, see:
#
# https://promptfoo.dev/docs/integrations/github-action
# https://github.com/promptfoo/promptfoo-action
#
name: LLM Prompt Evaluation

on:
  pull_request:
    paths:
      - 'prompts/**'

jobs:
  evaluate:
    runs-on: ubuntu-latest

    steps:
    - name: Checkout base ref (original)
      uses: actions/checkout@v2
      with:
        ref: ${{ github.base_ref }}
        path: base

    - name: Checkout head ref (modified)
      uses: actions/checkout@v2
      with:
        ref: ${{ github.head_ref }}
        path: head

    - name: Cache promptfoo data
      id: cache
      uses: actions/cache@v2
      with:
        path: ~/.cache/promptfoo
        key: ${{ runner.os }}-promptfoo-${{ hashFiles('**/package-lock.json') }}
        restore-keys: |
          ${{ runner.os }}-promptfoo-
        path: base

    - name: Checkout head ref (modified)
      uses: actions/checkout@v2
      with:
        ref: ${{ github.head_ref }}
        path: head

    - name: Cache promptfoo data
      id: cache
      uses: actions/cache@v2
      with:
        path: ~/.cache/promptfoo
        key: ${{ runner.os }}-promptfoo-${{ hashFiles('**/package-lock.json') }}
        restore-keys: |
          ${{ runner.os }}-promptfoo-

    - name: Set up Node.js
      uses: actions/setup-node@v2
      with:
        node-version: 16

    - name: Run promptfoo evaluation
      id: eval
      env:
        OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
        PROMPTFOO_CACHE_PATH: ~/.cache/promptfoo
      run: |
        npx promptfoo eval -c head/prompts/promptfooconfig.yaml --prompts base/prompts/prompt1.json head/prompts/prompt1.json -o output.json --share
        echo "OUTPUT_JSON_PATH=$GITHUB_WORKSPACE/output.json" >> $GITHUB_ENV

    - name: Comment PR
      uses: actions/github-script@v6
      with:
        github-token: ${{secrets.GITHUB_TOKEN}}
        script: |
          const fs = require('fs');
          const output = JSON.parse(fs.readFileSync(process.env.OUTPUT_JSON_PATH, 'utf8'));
          const body = `⚠️ LLM prompt was modified.\n\n| Success | Failure |\n|---------|---------|\n| ${output.results.stats.successes}      | ${output.results.stats.failures}       |\n\n**» [View eval results](${output.shareableUrl}) «**`;
          github.rest.issues.createComment({
            issue_number: context.issue.number,
            owner: context.repo.owner,
            repo: context.repo.repo,
            body: body
          });
